bitkeeper revision 1.1236.1.176 (424c2d3fy4zylRn8ctgTwLuwa-5b1Q)

Fix SMP TLB flushing in XenLinux 2.6.
Signed-off-by: Keir Fraser <keir@xensource.com>

diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
index 1d8f7814541a54f8e6db868b2403f6ddb3b3abee..002eb0abd1db9485c9134066fe029dd5d8faf05e 100644 (file)
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c
@@ -13,6 +13,7 @@
 #include <linux/pci.h>
 #include <linux/version.h>
 #include <asm/io.h>
+#include <asm/tlbflush.h>
 #include <asm-xen/balloon.h>
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
@@ -78,9 +79,9 @@ xen_contig_memory(unsigned long vstart, unsigned int order)
                phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
                        pfn+i;
        }
-       xen_tlb_flush();
+       flush_tlb_all();
 
-        balloon_unlock(flags);
+       balloon_unlock(flags);
 }
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)

diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c
index 7b7763848b92346e1a7b79df82523916c0133f6f..bcd398354e3968bbfae6f5fae47740f499170a88 100644 (file)
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c
@@ -203,6 +203,7 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
 
 #include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */
 
+#if 0 /* XEN */
 /*
  *     Smarter SMP flushing macros. 
  *             c/o Linus Torvalds.
@@ -441,6 +442,22 @@ void flush_tlb_all(void)
        on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
 }
 
+#else
+
+irqreturn_t smp_invalidate_interrupt(int irq, void *dev_id,
+                                    struct pt_regs *regs)
+{ return 0; }
+void flush_tlb_current_task(void)
+{ xen_tlb_flush_mask(current->mm->cpu_vm_mask); }
+void flush_tlb_mm(struct mm_struct * mm)
+{ xen_tlb_flush_mask(mm->cpu_vm_mask); }
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+{ xen_invlpg_mask(vma->vm_mm->cpu_vm_mask, va); }
+void flush_tlb_all(void)
+{ xen_tlb_flush_all(); }
+
+#endif /* XEN */
+
 /*
  * this function sends a 'reschedule' IPI to another CPU.
  * it goes straight through and wastes no time serializing

diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
index 1ac796d9fdaee33ef5bab903815bd91a129a36c7..82493c1236f1ce59c61174234c788945af1a24bc 100644 (file)
--- a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
+++ b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c
@@ -33,6 +33,7 @@
 #include <linux/vmalloc.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
+#include <asm/tlbflush.h>
 #include <asm-xen/hypervisor.h>
 #include <asm-xen/balloon.h>
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
@@ -94,6 +95,42 @@ void xen_invlpg(unsigned long ptr)
     BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 }
 
+#ifdef CONFIG_SMP
+
+void xen_tlb_flush_all(void)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_TLB_FLUSH_ALL;
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_tlb_flush_mask(cpumask_t mask)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_TLB_FLUSH_MULTI;
+    op.cpuset = mask.bits[0];
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_invlpg_all(unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_INVLPG_ALL;
+    op.linear_addr = ptr & PAGE_MASK;
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+void xen_invlpg_mask(cpumask_t mask, unsigned long ptr)
+{
+    struct mmuext_op op;
+    op.cmd = MMUEXT_INVLPG_MULTI;
+    op.cpuset = mask.bits[0];
+    op.linear_addr = ptr & PAGE_MASK;
+    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
+}
+
+#endif /* CONFIG_SMP */
+
 void xen_pgd_pin(unsigned long ptr)
 {
     struct mmuext_op op;
@@ -169,7 +206,7 @@ unsigned long allocate_empty_lowmem_region(unsigned long pages)
         phys_to_machine_mapping[__pa(vstart)>>PAGE_SHIFT] = INVALID_P2M_ENTRY;
     }
 
-    xen_tlb_flush();
+    flush_tlb_all();
 
     balloon_put_pages(pfn_array, 1 << order);
 

diff --git a/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c
index f73c2664b8b74335fb84bd0807388ef635e8dfce..649f64c402d25a9213a2e71bc112d0c50c705fd4 100644 (file)
--- a/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c
+++ b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c
@@ -260,8 +260,7 @@ static void balloon_process(void *unused)
 
         /* Ensure that ballooned highmem pages don't have cached mappings. */
         kmap_flush_unused();
-
-        xen_tlb_flush();
+        flush_tlb_all();
 
         /* No more mappings: invalidate pages in P2M and add to balloon. */
         for ( i = 0; i < debt; i++ )

diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h
index 714a247de381c7f412a8db895e041c023d0813f4..5333fde72bcc09690d1097536d68b9eaec476dd7 100644 (file)
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h
@@ -407,7 +407,8 @@ extern void noexec_setup(const char *str);
        do {                                                              \
                if (__dirty) {                                            \
                        if ( likely((__vma)->vm_mm == current->mm) ) {    \
-                           HYPERVISOR_update_va_mapping((__address), (__entry), UVMF_INVLPG_LOCAL); \
+                           HYPERVISOR_update_va_mapping((__address), (__entry), 0); \
+                           flush_tlb_page((__vma), (__address));         \
                        } else {                                          \
                             xen_l1_entry_update((__ptep), (__entry).pte_low); \
                            flush_tlb_page((__vma), (__address));         \

diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h
index 28fcf5a845c429970f50c5428d07019e82834d3e..4d13a650a2de88a564dd1df6ed10a7366becd4e4 100644 (file)
--- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h
@@ -6,23 +6,11 @@
 #include <asm/processor.h>
 
 #define __flush_tlb() xen_tlb_flush()
-
-/*
- * Global pages have to be flushed a bit differently. Not a real
- * performance problem because this does not happen often.
- */
 #define __flush_tlb_global() xen_tlb_flush()
+#define __flush_tlb_all() xen_tlb_flush()
 
 extern unsigned long pgkern_mask;
 
-# define __flush_tlb_all()                                             \
-       do {                                                            \
-               if (cpu_has_pge)                                        \
-                       __flush_tlb_global();                           \
-               else                                                    \
-                       __flush_tlb();                                  \
-       } while (0)
-
 #define cpu_has_invlpg (boot_cpu_data.x86 > 3)
 
 #define __flush_tlb_single(addr) xen_invlpg(addr)

diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
index 9bc3e4997bf822a8cc470ea40412b2b74e442a17..b50c5fc3ad95a242a83941ea608f4121f2a40a8d 100644 (file)
--- a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
+++ b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
@@ -83,6 +83,14 @@ void xen_pte_unpin(unsigned long ptr);
 void xen_set_ldt(unsigned long ptr, unsigned long bytes);
 void xen_machphys_update(unsigned long mfn, unsigned long pfn);
 
+#ifdef CONFIG_SMP
+#include <linux/cpumask.h>
+void xen_tlb_flush_all(void);
+void xen_invlpg_all(unsigned long ptr);
+void xen_tlb_flush_mask(cpumask_t mask);
+void xen_invlpg_mask(cpumask_t mask, unsigned long ptr);
+#endif
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
 /* 
 ** XXX SMH: 2.4 doesn't have percpu.h (or support SMP guests) so just